import json
import logging
import os
import stat
import time
import uuid
import paramiko
import queue
import threading
from service import DBSql


class SCPClient():
    def __init__(self, task_id, ip_id):
        self.db = DBSql()
        self.task_id = task_id
        self.ip_id = ip_id
        self.dir_queue = queue.Queue()
        self.download_file_queue = queue.Queue()
        self.sql_insert_queue = queue.Queue()
        self.service = 'scp'
        sql = "SELECT * FROM `ip_list_%s` WHERE `id` = %s"
        with self.db.query(sql_text=sql, params=(self.task_id, self.ip_id)) as info:
            pass

        if info:
            config = info.get(f'{self.service}_config')
            config = json.loads(config)
            self.hostname = config.get('hostname')
            self.port = int(config.get('port'))
            self.username = config.get('username')
            self.password = config.get('password')
            self.paths = config.get('path')
            for path in self.paths:
                self.dir_queue.put(path)
            
        self.task_uuid = str(uuid.uuid4())
        self.task_folder_path = './save_files/' + self.task_uuid
        if not os.path.exists(self.task_folder_path):
            os.makedirs(self.task_folder_path)
        self.save_file_type = ['pdf', 'doc', 'docx', 'txt']
        self.scaned_path = set()
        self.downloaded_file = set()
        self.scaned_path_nums = 0
        self.thread_nums = 10
        self.thread_running_flag = {} # 用于标记线程是否在运行
        self.skip_path = ['/proc', '/sys', '公司/artical_scan', '.local/share/Trash', '.venv', '/smb-share']
        self.sql_insert_num = 20
        self.can_connect = True
    
    def create_client(self):
        try:
            client = paramiko.SSHClient()
            client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            client.connect(hostname=self.hostname, username=self.username, password=self.password, port=self.port, timeout=10)
            scp = paramiko.SFTPClient.from_transport(client.get_transport())
            return client, scp
        except Exception as e:
            logging.error(e)
            logging.error(f'进程运行情况：{self.thread_running_flag.values()}')
            self.can_connect = False
            return None, None
    
    def save_file_info(self, file_name, file_uuidname, file_save_path, file_path, file_format, file_size):
        sql = """
            INSERT INTO `file_info_%s` 
            (`file_name`, `file_uuidname`, `file_save_path`, `file_path`, `file_format`, `file_size`, `service`, `ip_id`, `port`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
        """
        with self.db.insert(sql_text=sql, params=(self.task_id, file_name, 
                                                  file_uuidname, file_save_path, 
                                                  file_path, file_format, file_size,
                                                    self.service, self.ip_id, self.port)):
            pass
    
    def save_file_info_many(self, data:list):
        # file_name, file_uuidname, file_save_path, file_path, file_format, file_size = data[0]
        data_len = len(data)
        sql = """
            INSERT INTO `file_info_%s` 
            (`file_name`, `file_uuidname`, `file_save_path`, `file_path`, `file_format`, `file_size`, `service`, `ip_id`, `port`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
        """+",(%s, %s, %s, %s, %s, %s, %s, %s, %s)"*(data_len-1)
        params = [self.task_id]
        for item in data:
            item.extend([self.service, self.ip_id, self.port])
            params.extend(item)
        with self.db.insert(sql_text=sql, params=tuple(params)):
            pass

    def download_file(self, remote_path, file_size, scp):
        logging.info(f'下载文件：【{remote_path}】中...')
        file_name = remote_path.split('/')[-1]
        file_format = file_name.split('.')[-1]
        file_uuidname = str(uuid.uuid4()) + '.' + file_format
        local_path = os.path.join(self.task_folder_path,file_uuidname)
        try:
            scp.get(remote_path, local_path)
            self.sql_insert_queue.put([file_name, file_uuidname, local_path, remote_path, file_format, file_size])
            # self.save_file_info(file_name, file_uuidname, local_path, remote_path, file_format, file_size)
            self.downloaded_file.add(remote_path)
            return True
        except Exception as e:
            if 'Broken pipe' in str(e):
                self.can_connect = False
                logging.error(f'{remote_path}：连接已断开，请检查网络')
                return False
            logging.error(f'{remote_path}：' + str(e))
        return False
    
    def is_dir(self, remote_path, scp):
        file_size = 0
        try:
            file_size = scp.lstat(remote_path).st_size
            st_mode = scp.lstat(remote_path).st_mode
            if stat.S_ISDIR(st_mode):
                return 'dir', file_size
            elif stat.S_ISLNK(st_mode):
                return 'link', file_size
            return 'file', file_size
        except IOError:
            return False, file_size
    
    def get_files(self, remote_path, scp):
        logging.info(f'扫描文件夹：【{remote_path}】中...')
        for path_ in self.skip_path:
            if path_ in remote_path:
                return
        self.scaned_path.add(remote_path)
        for file in scp.listdir(remote_path):
            file_path = os.path.join(remote_path, file)
            flag, file_size = self.is_dir(file_path, scp)
            if flag == 'dir' and file_path not in self.scaned_path:
                self.dir_queue.put(file_path)
            elif flag == 'file':
                file_format = file.split('.')[-1]
                if file_format in self.save_file_type and file_path not in self.downloaded_file:
                    self.download_file_queue.put((file_path, file_size))
                    # self.download_file(file_path, file_size)
        self.scaned_path_nums += 1
    
    def is_any_thread_running(self):
        # logging.error(self.thread_running_flag.values())
        for status in self.thread_running_flag.values():
            if status:
                return True
        return False

    def scan_files(self):
        client, scp = self.create_client()
        thread_name = str(uuid.uuid4())
        self.thread_running_flag[thread_name] = False
        while (self.is_any_thread_running() or not self.dir_queue.empty()) and self.can_connect:
            try:
                try:
                    dir_path = self.dir_queue.get(timeout=0.1)
                except:
                    continue
                self.thread_running_flag[thread_name] = True
                self.get_files(dir_path, scp)
            except Exception as e:
                if 'Broken pipe' in str(e):
                    self.can_connect = False
                    logging.error(f'连接已断开，请检查网络')
                    return False
                logging.error(f'{dir_path}：' + str(e))
            self.thread_running_flag[thread_name] = False
            if not self.scaned_path_nums % 500:
                logging.warning('-'*80)
                logging.warning(f'待扫描文件夹数量：{self.dir_queue.qsize()}')
                logging.warning(f'已扫描文件夹数量：{self.scaned_path_nums}')
                logging.warning(f'已下载文件数量：{len(self.downloaded_file)}')
                logging.warning(f'进程运行情况：{self.thread_running_flag.values()}')
                logging.warning('-'*80)
        client.close()
    
    def download_file_thread(self):
        client, scp = self.create_client()
        thread_name = str(uuid.uuid4())
        self.thread_running_flag[thread_name] = False
        while (self.is_any_thread_running() or not self.dir_queue.empty() or not self.download_file_queue.empty()) and self.can_connect:
            try:
                try:
                    data = self.download_file_queue.get(timeout=0.1)
                except:
                    continue
                self.thread_running_flag[thread_name] = True
                self.download_file(data[0], data[1], scp)
            except Exception as e:
                logging.error(f'Download【{data[0]}】Error, Filesize【{data[1]}】：' + str(e))
            self.thread_running_flag[thread_name] = False
        client.close()
    
    def sql_insert_thread(self):
        thread_name = str(uuid.uuid4())
        self.thread_running_flag[thread_name] = False
        while (self.is_any_thread_running() or not self.dir_queue.empty() \
            or not self.download_file_queue.empty() or not self.sql_insert_queue.empty()) and self.can_connect:
            data_list = []
            try:
                for _ in range(self.sql_insert_num):
                    try:
                        data = self.sql_insert_queue.get(timeout=0.1)
                        data_list.append(data)
                    except:
                        continue
                if not data_list:
                    continue
                self.thread_running_flag[thread_name] = True
                self.save_file_info_many(data_list)
            except Exception as e:
                logging.error(f'sql_insert_thread：' + str(e))
            self.thread_running_flag[thread_name] = False
            
        
            
def main():
    css = SCPClient(task_id=1, ip_id=1)
    # 我有办法了！ 遍历和下载的时候我直接创建新客户端遍历
    # 我真是个天才
    threads = []
    threads.append(threading.Thread(target=css.download_file_thread))
    threads.append(threading.Thread(target=css.download_file_thread))
    threads.append(threading.Thread(target=css.sql_insert_thread))
    for _ in range(css.thread_nums):
        threads.append(threading.Thread(target=css.scan_files))

    
    start_time = time.time()
    for t in threads:
        t.start()
        time.sleep(0.5)

    for t in threads:
        t.join()
    end_time = time.time()
    logging.warning(f'耗时：{end_time - start_time}秒')
    logging.warning('-'*80)
    logging.warning(f'待扫描文件夹数量：{css.dir_queue.qsize()}')
    logging.warning(f'已扫描文件夹数量：{css.scaned_path_nums}')
    logging.warning(f'已下载文件数量：{len(css.downloaded_file)}')
    logging.warning(f'进程运行情况：{css.thread_running_flag.values()}')
    logging.warning(f'连接情况：{css.can_connect}')
    logging.warning('-'*80)

### 本大聪明只能尝试在一个类里搞n个客户端了，奶奶滴！！
if __name__ == '__main__':
    main()
    # css = SCPClient(task_id=1, ip_id=1)
    # client, scp = css.create_client()
    # dir_list = scp.listdir('/')
    # print(dir_list)
    # client.close()
